package com.chance.cc.crawler.prod.command.meta.ifeng;

import com.chance.cc.crawler.meta.core.bean.crawler.CrawlerDomainKeys;
import com.chance.cc.crawler.prod.command.meta.CommonCrawlerKeywordScript;

import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.*;
import java.util.stream.Collectors;

/**
 * @ClassName song
 * @Description TODO
 * @Author ding
 * @Date 2021/9/3 16:11
 * @Version 1.0
 **/
public class ifengKeywordScript extends CommonCrawlerKeywordScript {
    private static final String domain = "ifeng";

    public static void main(String[] args) {
        series();
    }

    private static void series() {
        String site = "article";
        String filePath = "E:\\chance-crawler-development\\crawler-scripts-command-prod\\src\\main\\java\\com\\chance\\cc\\crawler\\prod\\command\\meta\\ifeng\\ifeng.txt";
        List<String> keywords = duplicateKeywords(filePath);
        //addKeywords(site,keywords);
        delKeywords(site,keywords);
    }
    private static void delKeywords(String site,List<String> keywords){
        for (String keyword : keywords) {
            System.out.println(keyword);
            CrawlerDomainKeys crawlerDomainKeys = new CrawlerDomainKeys();
            crawlerDomainKeys.setDomain(domain);
            crawlerDomainKeys.setSite(site);
            crawlerDomainKeys.setKeyword(keyword);
            crawlerDomainKeys.setStatus(0);
            keywordCommand.delKeys(crawlerDomainKeys);
        }
    }

    private static List<String> duplicateKeywords(String filePath){
        Set<String> duplicate = new HashSet<>(getContent(filePath));
        return new ArrayList<>(duplicate);
    }
    public static List<String> getContent(String filepath)  {
        //读取文件
        List<String> lineLists = null;
        try {
            System.out.println(filepath);
            lineLists = Files
                    .lines(Paths.get(filepath), Charset.defaultCharset())
                    .flatMap(line -> Arrays.stream(line.split("\n")))
                    .collect(Collectors.toList());
            lineLists.removeIf(String::isEmpty);
        } catch (IOException e) {
            e.printStackTrace();
        }
        return  lineLists;
    }
}
